'''
* This is the projet for Brtc LlmOps Platform
* @Author Leon-liao <liaosiliang@alltman.com>
* @Description //TODO 
* @File: 5_self_documeny_loader.py
* @Time: 2025/8/20
* @All Rights Reserve By Brtc
'''
# from typing import Iterator, AsyncIterator
# from langchain_core.document_loaders import BaseLoader
# from langchain_core.documents import Document
# class CustomDocumentLoader(BaseLoader):
#     """自定义文档加载器"""
#     def __init__(self, file_path: str)->None:
#         self.file_path = file_path
#
#     def lazy_load(self)->Iterator[Document]:
#         """逐行读取文件数据， 并使用 yield返回文档"""
#         with open(self.file_path, encoding="utf-8") as f:
#             line_number = 0
#             for line in f:
#                 yield Document(page_content=line, metadata={"line_number": line_number, "source":self.file_path})
#                 line_number += 1
#
#     async def  alazy_load(self) -> AsyncIterator[Document]:
#         import aiofiles
#         async with aiofiles.open(self.file_path, encoding="utf-8") as f:
#             line_number = 0
#             async for line in f:
#                 yield Document(page_content=line, metadata={"line_number": line_number, "source":self.file_path})
#                 line_number += 1
#
# loader = CustomDocumentLoader("./eshop_goods.txt")
# documents = loader.load()
#
# print(documents)
# print(len(documents))
# print(documents[0].metadata)

from typing import Iterator, AsyncIterator
from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import  Document


class CustomDocumentLoader(BaseLoader):
    def __init__(self,file_path:str)->None:
        self.file_path = file_path

    def lazy_load(self)->Iterator[Document]:
        with open(self.file_path,encoding="utf-8") as f:
            line_number = 0
            for line in f:
                yield Document(page_content=line,metadata={"line_number":line_number,"source":self.file_path})
                line_number += 1

    async def alazy_load(self) -> AsyncIterator[Document]:
        import aiofiles
        async with aiofiles.open(self.file_path,encoding="utf-8") as f:
            line_number = 0
            async for line in f:
                yield Document(page_content=line,metadata={"line_number":line_number,"source":self.file_path})
                line_number += 1

loader = CustomDocumentLoader("./eshop_goods.txt")
documents =loader.load()

print(documents)
print(len(documents))
print(documents[0].metadata)